# Updated by Wei 2013/08/05 morning by Wei at school

# Updated by Wei 2013/08/04 night by Wei at school
# Fit for latest 3 factor probabilities formula
# Program Purpose: This program is to compute the threshold for the probability of the overall importance based on some existing histogram files.

# Updated by Wei 2013/06/19
# Note: N/A

print "Program begins..."
print "This program is to compute the threshold for the probability of the overall importance based on an existing histogram file."
print "It generates threshold for each component of our formula OR the combinations of our components"
# For the machine dodo:
# option0
# inputFileName = "/home/diaosi/outputDirForTermScores/histogram.txt"
# option1
# inputFileName = "/home/diaosi/outputDirForTermScores/histogram_sorted_by_first_column.txt"
# option2
# inputFileName = "/home/diaosi/outputDirForTermScores/histogram_all_sorted_by_first_column.txt"
# option3
# inputFileName = "/home/diaosi/outputDirForTermScores/histogram_all_1000000TimesProbability_sorted_by_first_column_probability_increasing.txt"
# option4
# inputFileName = "/home/diaosi/outputDirForTermScores/histogram_all_1000000TimesProbability_by_our_2D_prediction_sorted_by_first_column_probability_increasing.txt"

# For the machine pangolin:
# option0 --- the set of full histogram files without certain thresholds.
# The rows are needed to be sorted in increasing order of their assigned probability
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_ALL_1000000Times_Posting_Probability_given_query_terms_SORTED_by_increasing_order_of_probability.txt"
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_ALL_1000000Times_Posting_Probability_NOT_given_anything_using_true_query_term_probability_distribution_SORTED_by_increasing_order_of_probability.txt"
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_ALL_1000000Times_Posting_Probability_NOT_given_anything_using_our_1D_query_term_probability_distribution_SORTED_by_increasing_order_of_probability.txt"
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_ALL_1000000Times_Posting_Probability_NOT_given_anything_using_our_2D_query_term_probability_distribution_SORTED_by_increasing_order_of_probability.txt"
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_ALL_1000000Times_Posting_Probability_NOT_given_anything_using_good_turing_query_term_probability_distribution_SORTED_by_increasing_order_of_probability.txt"
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_soalr_original_Three_Probabiliy_Factors_Combined.txt.deubg"
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_original_1_2_3_Probabiliy_Factors_Combined_weiTestingQuery.DEBUG.LEAVE"

print "inputFileName:",inputFileName
inputFileHandler = open(inputFileName,"r")

TOTAL_NUM_OF_POSTINGS = 0
previousScore = 0.0
currentScore = 0.0
previousScoreFreq = 0
currentScoreFreq = 0

# step1:
# This is the FIRST pass
# Compute the total number of postings we have sampled
previousLine = ""
currentLine = inputFileHandler.readline()
while currentLine:
    currentScore = float(currentLine.strip().split(" ")[0])
    currentScoreFreq = int(currentLine.strip().split(" ")[1])
    
    # print "currentLine:",currentLine
    if previousScore <= currentScore:
        pass
    else:
        print "The scores are NOT strictly in increasing order"
        print "previousLine:",previousLine.strip()
        print "currentLine:",currentLine.strip()
        exit(1)
    
    TOTAL_NUM_OF_POSTINGS += currentScoreFreq
    
    previousLine = currentLine
    previousScore = currentScore
    previousScoreFreq = currentScoreFreq
    
    currentLine = inputFileHandler.readline()

print "TOTAL_NUM_OF_POSTINGS among the sampling query terms:",TOTAL_NUM_OF_POSTINGS
inputFileHandler.close()


# step2:
# This is the SECOND pass
# Purpose: compute the threshold given the percentageOfTotalPostingsToCut
print "--->Output Begins..."
print "universalCuttingThreshold percentageToKeepOfTheWholeIndex percentageOfTotalPostingsToCut totalNumOfPostingsAmongTheSamplingQueryTerms currentTotalNumOfPostingsWillBeCutAmongTheSamplingQueryTerms"
outputingLine = "#" + " " + str(0.0) + "                   " + str(1.0) + "                   " + str(0.0) + "                   " + str(TOTAL_NUM_OF_POSTINGS) + "                   " + str(0)
print outputingLine
percentageOfTotalPostingsToCutList = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99]
for listValue in percentageOfTotalPostingsToCutList:
    percentageOfTotalPostingsToCut = listValue
    percentageOfTotalPostingsToKeep = 1 - percentageOfTotalPostingsToCut
    
    inputFileHandler = open(inputFileName,"r")
    total_num_of_postings_being_cut = int( TOTAL_NUM_OF_POSTINGS * percentageOfTotalPostingsToCut )
    

    
    currentLine = inputFileHandler.readline()
    currentTotalNumOfPostingsWillBeCut = 0
    
    currentLineScore = 0.0
    currentLineFreq = 0
    
    while currentLine:
        currentLineScore = float( currentLine.strip().split(" ")[0] )
        currentLineFreq = int( currentLine.strip().split(" ")[1] )
        currentTotalNumOfPostingsWillBeCut += currentLineFreq
        if currentTotalNumOfPostingsWillBeCut >= total_num_of_postings_being_cut:
            break
        currentLine = inputFileHandler.readline()
    
    # for debug ONLY (priority level = HIGH)
    '''
    print "percentageOfTotalPostingsToCut:",percentageOfTotalPostingsToCut
    print "TOTAL_NUM_OF_POSTINGS:",TOTAL_NUM_OF_POSTINGS
    print "total_num_of_postings_being_cut among the sampling query terms:",total_num_of_postings_being_cut    
    print "currentTotalNumOfPostingsWillBeCut among the sampling query terms:",currentTotalNumOfPostingsWillBeCut
    print "currentLineScore(threshold):",currentLineScore
    print "currentLineFreq:",currentLineFreq
    print "currentLine:",currentLine.strip()
    '''
    
    outputingLine = "#" + " " + str(currentLineScore) + "                   " + str(percentageOfTotalPostingsToKeep) + "                   " + str(percentageOfTotalPostingsToCut) + "                   " + str(TOTAL_NUM_OF_POSTINGS) + "                   " + str(currentTotalNumOfPostingsWillBeCut)
    print outputingLine
    inputFileHandler.close()
print "--->Output Ends."
print "Program ends."

